package com.vint.nee.processor;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.pipeline.ConsolePipeline;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.PlainText;
import us.codecraft.webmagic.selector.Selectable;
import us.codecraft.webmagic.selector.XpathSelector;

import java.util.ArrayList;
import java.util.List;

/**
 * Copyright (C) 2017, Xiaomi Inc. All rights reserved.
 * Created by vint @ 2017/10/29.
 * in auction
 */


//瑞平国际

public class RpgjProcessor implements PageProcessor{

	private Site site = Site.me().setDomain("www.ruiping.com");

	private static final String url = "http://www.ruiping.com/yuzhan/";

	@Override
	public void process(Page page){
		List<String> links = page.getHtml().links().regex("list_30_\\d+").all();
		List<String> targets = new ArrayList<>();
		for(String s:links){
			targets.add(url+s+".html");
		}
		page.addTargetRequests(targets);

		List<Selectable> lis =  page.getHtml().xpath("//ul[@class='s62']/li").nodes();

		for(Selectable s :lis){
			String imgUrl = url+s.xpath("//a/img/@src").toString();
			String href = url+s.xpath("//a/@href").toString();
			String title = s.xpath("//a/@title").toString();
			System.out.println("title:"+title);
			System.out.println("imgUrl:"+imgUrl);
			System.out.println("href:"+href);
		}
		//TODO:
		//1.把上面的text存储下来，图片用ImageCommon.java下载下来
		//2.把取得的数据导出成excel

	}

	@Override
	public Site getSite(){
		return site;
	}
	public static void main(String[] args) {
		Spider.create(new RpgjProcessor()).addUrl(url).addPipeline(new ConsolePipeline()).run();

	}
}
